<?php
	require_once 'StringCache.class.php';
	require_once 'WorkingMemory.class.php';

	class Processor {
		private $DB, $Config, $SA, $Cache, $Memory, $ResultSet, $Contributions;
		private $rate_of_change, $prev_time, $prev, $fraud_levels, $l_fraud_levels, $affected_news;
		private $fraud_threshold, $feedback_weight, $feedback_exponent;
		
		function __construct() {
			global $Config;
			date_default_timezone_set($Config -> Root['Processor']['Timezone']);
			
			$this -> fraud_threshold = $Config -> Root['Magic']['FraudThreshold'];
			$this -> feedback_weight = $Config -> Root['Magic']['FeedbackWeight'];
			$this -> feedback_exponent = $Config -> Root['Magic']['FeedbackExponent'];
			
			$this -> ResultSet = array();
			$this -> Contributions = array();
			$this -> Config = $Config;
			$this -> Cache = new StringCache($Config -> Root['Semantic']['CacheMax']);
			$this -> DB = new Database();
			$this -> SA = new SemanticAnalyser();
			$this -> SA -> readScript(realpath('../Semantic/semantic_rules.txt'), $Config -> Root['Semantic']['DefaultWidth']);
			
			$this -> Memory = new WorkingMemory();
			$this -> build_working_memory();
		}
		
		private function apply_feedback($since) {
			$feedback = array();
			$sql = $this -> DB -> query("
				SELECT *
				FROM `Events`, `Events_Feedback`
				WHERE `Events`.`Timestamp` > '$since'
				AND `Events`.`ID` = `Events_Feedback`.`EventID`
				ORDER BY `Events`.`Timestamp` DESC
			");
			
			while ($row = $this -> DB -> row($sql)) {
				$feedback[$row['Symbol']][] = $row['Score'];
			}
			
			$this -> l_fraud_levels = $this -> fraud_levels;
			
			$symbols = array_keys($this -> rate_of_change);
			foreach ($symbols as $symbol) {
				if (!isset($feedback[$symbol])) continue;
				foreach ($feedback[$symbol] as $i => $score) {
					$n = $i + 1;
					foreach (array('Ask', 'Volume') as $k) {
						$this -> l_fraud_levels[$s][$k] += ($this -> feedback_weight / ($n * $this -> feedback_exponent)) * $score;
					}
				}
			}
		}
		
		private function array_add($a, $b) {
			foreach ($b as $k => $v) {
				if (isset($a[$k])) $a[$k] += $b[$k];
				else $a[$k] = $b[$k];
			}
			
			return $a;
		}
		
		private function build_working_memory() {
			echo "Calculating company semantic scores...\n";
			$sql = $this -> DB -> select('*', 'Processed_News', false, false, 'Timestamp', 'ASC');
			$prev = false;
			
			while ($row = $this -> DB -> row($sql)) {
				$scores = json_decode($row['Semantics']);
				foreach ($scores as $sym => $score) {
					if (!isset($prev[$sym])) $prev[$sym] = $score;
					else {
						$scores[$sym] = $prev[$sym] + $score;
						$prev[$sym] = $scores[$sym];
					}
				}
				
				$this -> ResultSet[$row['Timestamp']] = $this -> array_add($this -> ResultSet[$row['Timestamp']], $scores);
				$keys = array_keys($scores);
				
				foreach ($keys as $key) {
					$this -> Memory -> increment('Semantics', $key, $scores[$key]);
					$this -> Contributions[$key][$row['ID']] = $scores[$key];
				}
			}
			
			$this -> calculate_fraud_levels();
		}
		
		private function calculate_cutoff_date() {
			$stamp = strtotime('today');
			$days = $this -> Config -> Root['Processor']['OldThreshold'];
			$weekends = floor($days / 5);
			$days += $weekends * 2;
			$stamp -= $days * 24 * 3600;
			
			return $stamp;
		}
		
		private function calculate_fraud_levels() {
			echo "Rebuilding fraud levels...\n";
			$sql = $this -> DB -> select(array('Symbol', 'Ask', 'Volume', 'Timestamp'), 'Processed_Ftse', false, false, 'Timestamp', 'ASC');
			
			$this -> rate_of_change = array();
			$this -> prev_time = array();
			$this -> prev = array();
			$times = array_keys($this -> ResultSet);
			$earliest = 0;
			$this -> fraud_levels = array();
			$this -> l_fraud_levels = array();
			
			while ($row = $this -> DB -> row($sql)) {
				if (!$earliest) $earliest = $row['Timestamp'];
				$news_found = false;
				$s = $row['Symbol'];
				if (!isset($this -> prev_time[$s])) $this -> prev_time[$s] = $row['Timestamp'];
				
				$news_found = $this -> calculate_rate_of_change($s, $row, $times);
				
				if ($news_found) {
					$time = array_shift($times);
					unset($this -> ResultSet[$time][$s]);
				}
			}
			
			$this -> apply_feedback($earliest);
		}
		
		private function calculate_rate_of_change($s, $row, $history = false) {
			$used_history = false;
			
			foreach (array('Ask', 'Volume') as $k) {
				if (!isset($this -> rate_of_change[$s][$k])) {
					$this -> rate_of_change[$s][$k] = $row[$k];
					$this -> prev[$s][$k] = $row[$k];
					$this -> prev_time[$s] = $row['Timestamp'];
					continue;
				}
				
				$this -> rate_of_change[$s][$k] =
					$this -> rate_of_change[$s][$k] * 0.9
					* ($row[$k] - $this -> prev[$s][$k]) * 0.1
					/ (($this -> prev_time[$s] == $row['Timestamp']) ? 1 : ($row['Timestamp'] - $this -> prev_time[$s]));

				if (!$history) {
					if (isset($this -> affected_news[$s]) && $this -> affected_news[$s]) {
						$this -> affected_news[$s] = false;
						$this -> rate_of_change[$s][$k] += $this -> Memory -> recall('Semantics', $s);
					}
				} else {
					if (current($history) <= $row['Timestamp'] && isset($this -> ResultSet[current($history)][$s])) {
						$this -> rate_of_change[$s][$k] += $this -> ResultSet[current($history)][$s];
						$used_history = true;
					}
				}
				
				$this -> fraud_levels[$s][$k] = ($row[$k] - $this -> prev[$s][$k]) / (($this -> rate_of_change[$s][$k] != 0) ? $this -> rate_of_change[$s][$k] : 1);
			}
			
			return $used_history;
		}
		
		private function cleanup() {
			echo "Cleaning up old data...\n";
			$stamp = $this -> calculate_cutoff_date();
			
			foreach (array('Processed_News', 'Processed_Ftse') as $tbl) {
				if ($tbl === 'Processed_News') {
					$sql = $this -> DB -> select('*', $tbl, "Timestamp < $stamp");
					while ($row = $this -> DB -> row($sql)) {
						$scores = json_decode($row['Semantics']);
						$keys = array_keys($scores);
						
						foreach ($keys as $key) {
							$this -> Memory -> decrement('Semantics', $key, $scores[$key]);
							unset($this -> Contributions[$key][$row['ID']]);
						}
					}
				}
				$sql = $this -> DB -> deleteWhere($tbl, "Timestamp < $stamp");
			}
			
			$this -> calculate_fraud_levels();
		}
		
		private function normalise($diff, $range, $min, $max) {
			$quotient = ($max - $min) / $range;
			return round($diff * $quotient) + $min;
		}
		
		public function process() {
			$processed = array();
			$delete = array();
			$events = array();
			
			$sql = $this -> DB -> select('*', 'Unprocessed_News');
			$i = 0;
			while ($row = $this -> DB -> row($sql)) {
				$delete[] = $row['ID'];
				if ($row['Title'] === '' && $row['Body'] === '') continue;
				$processed[$i] = $row;
				unset($processed[$i]['ID']);
				
				// Utilise a cache as there are lots of duplicate news items
				// and it doesn't make sense to semantically parse them if they
				// have already been parsed.
				if (!$this -> Cache -> exists($row['Title'])) {
					$semantic_score = $this -> SA -> parse($row['Title']);
					$semantic_score = $this -> array_add($semantic_score, $this -> SA -> parse($row['Body']));
					$semantic_score = json_encode($semantic_score);
					$this -> Cache -> add($row['Title'], $semantic_score);
				} else {
					$semantic_score = $this -> Cache -> get($row['Title']);
				}
				
				$processed[$i]['Semantics'] = $semantic_score;
				
				$i++;
			}
			
			if ($i > 0) {
				$this -> DB -> insert($processed, 'Processed_News', true);
				$this -> DB -> delete('ID', $delete, 'Unprocessed_News');
				echo "Processed $i news items.\n";
			}
			
			$date_start = date('Y-m-d H:i:s', time() - 2); // ZOMG magic constant, so naughty
			
			// Select the recently added news items (if any)
			$sql = $this -> DB -> select('*', 'Processed_News', "TimeInserted > '$date_start'");
			$new_news = $this -> DB -> count($sql);
			echo "Selected $new_news new news items.\n";
			while ($row = $this -> DB -> row($sql)) {
				$scores = json_decode($this -> Cache -> get($row['Title']));
				$this -> Memory -> Store['Semantics'] = $this -> array_add(
					(isset($this -> Memory -> Store['Semantics'])
					? $this -> Memory -> Store['Semantics']
					: array())
					, $scores
				);
				foreach ($scores as $sym => $score) {
					$this -> Contributions[$sym][$row['ID']] = $score;
					$this -> affected_news[$sym] = true;
				}
			}
			
			$processed = array();
			$delete = array();
			$sql = $this -> DB -> select('*', 'Unprocessed_Ftse');
			$i = 0;
			
			while ($row = $this -> DB -> row($sql)) {
				$s = $row['Symbol'];
				$this -> calculate_rate_of_change($s, $row);
				
				$processed[$i] = $row;
				$delete[] = $row['ID'];
				unset($processed[$i]['ID']);
				$i++;
			}
			
			if ($i > 0) {
				$this -> DB -> insert($processed, 'Processed_Ftse');
				$this -> DB -> delete('ID', $delete, 'Unprocessed_Ftse');
				echo "Processed $i FTSE100 items.\n";
			}
			
			$this -> apply_feedback($this -> calculate_cutoff_date());
			foreach ($this -> l_fraud_levels as $sym => $levels) {
				foreach ($levels as $stat => $level) {
					if ($level > $this -> fraud_threshold || $level < $this -> fraud_threshold * -1) {
						$med = $this -> Config -> Root['Magic']['SeverityMed'] * $this -> fraud_threshold;
						$high = $this -> Config -> Root['Magic']['SeverityHigh'] * $this -> fraud_threshold;
						$alert_level = 'LOW';
						
						if ($level > $high || $level < $high * -1) {
							if ($level > $high * 2) $level = $high;
							if ($level < $high * -2) $level = $high * -1;
							$range = $high;
							$diff = ($level > $high) ? $level - $high : ($high * -1) - $level;
							$severity = $this -> normalise($diff, $range, 170, 255);
							$alert_level = 'HIGH';
						} elseif ($level > $med || $level < $med * -1) {
							$range = $high - $med;
							$diff = ($level > $med) ? $level - $med : ($med * -1) - $level;
							$severity = $this -> normalise($diff, $range, 85, 169);
							$alert_level = 'MED';
						} else {
							$range = $med - $this -> fraud_threshold;
							$diff = ($level > $this -> fraud_threshold) ? $level - $this -> fraud_threshold : $this -> fraud_threshold - $level;
							$severity = $this -> normalise($diff, $range, 0, 84);
						}
						
						echo "Potential fraud of severity $alert_level detected in the $stat value for $sym. Sending alerts...\n";
						
						// Very heavy database work here, not sure if can be optimised easily
						$this -> DB -> insert(
							array(
								'Severity' => $severity,
								'Timestamp' => time(),
								'Symbol' => $sym,
								'Stat' => $stat
							),
							'Events'
						);
						
						$eid = $this -> DB -> getId();
						$time_from = $this -> calculate_cutoff_date();
						$time_to = time();
						
						$this -> DB -> insert(
							array(
								'EventID' => $eid,
								'TimeFrom' => $time_from,
								'TimeTo' => $time_to
							),
							'Events_Ftse_Range'
						);
						
						$sql = $this -> DB -> select('*', 'Processed_Ftse', "Timestamp < $time_to AND Timestamp > $time_from");
						$insertions = array();
						while ($row = $this -> DB -> row($sql)) {
							$insertions[] = $row;
						}
						
						$this -> DB -> insert($insertions, 'Storage_Ftse', true);
						
						if (isset($this -> Contributions[$sym])) {
							$nids = array();
							$insertions = array();
							foreach ($this -> Contributions[$sym] as $nid => $score) {
								$nids[] = $nid;
								$insertions[] = array(
									'EventID' => $eid,
									'NewsID' => $nid,
									'Score' => $score
								);
							}
							
							$this -> DB -> insert($insertions, 'Events_News');
							$nids = Database::commaSeparate($nids, false);
							$sql = $this -> select('*', 'Processed_News', "`ID` IN ($nids)");
							$insertions = array();
							while ($row = $this -> DB -> row($sql)) {
								unset($row['ID'], $row['Semantics'], $row['TimeInserted']);
								$insertions[] = $row;
								
							}
							
							$this -> DB -> insert($insertions, 'Storage_News', true);
						}
					}
				}
			}
			
			$this -> cleanup(); // Could possibly be a separate process but not necessary
		}
	}
?>